package org.newlucene.core.analysis;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class WhiteSpaceAnalyzer implements Analyzer
{
	private String regexp = "[^\\s]+";
	private Pattern pattern = Pattern.compile(regexp);
	private Matcher matcher;

	// always converts tokens to lower case before returning
	public WhiteSpaceAnalyzer()
	{
		this.matcher = pattern.matcher("");
	}

	public void set( String text)
	{
		this.matcher.reset(text);
	}

	public Token next()
	{
		if (matcher.find())
		{
			String text = matcher.group();
			return new Token(text.toLowerCase());
		} else
		{
			return null;
		}
	}
	
	public static void main(String[] args)
	{
		WhiteSpaceAnalyzer a = new WhiteSpaceAnalyzer();
		a.set("hello world it works now!");
		Token t;
		while ((t = a.next()) != null)
		{
			System.out.println(t);
		}
	}
}